﻿import MySQLdb
import funcs
url_set=[0]*(2**funcs.SEG)
for i in xrange(len(url_set)):
    url_set[i]=[0]*(2**funcs.SEG_OFFSET)
def set_url_mark(l_md5):
    seg_index,seg_offset,offset_value=funcs.get_bit_size(l_md5)
    u_offset_value=url_set[seg_index][seg_offset]
    b_u_o_b=(u_offset_value>>offset_value)%2
    if b_u_o_b==0:
        u_offset_value=u_offset_value|(1<<offset_value)
        url_set[seg_index][seg_offset]=u_offset_value
        return_value=True
    else:
        return_value=False
    return return_value

cxn=MySQLdb.connect(db='pachong',host='localhost',user='root',passwd='123456ms')
cur=cxn.cursor()

sql='select min(id) from csdn'
cur.execute(sql)
min_id=cur.fetchone()[0]

sql='select max(id) from csdn'
cur.execute(sql)
max_id=cur.fetchone()[0]

step=2000
#max_id=69000
idx=min_id
while idx<=max_id:
    sql='select * from csdn where id between %s and %s'
    cur.execute(sql,(str(idx),str(idx+step)))
    for item in cur.fetchall():
        _id=item[0]
        _url=item[1]
        if not set_url_mark(_id):
            print _id
    idx+=(step+1)
def check_url_mark_num():
    mark_num=0
    for seg in url_set:
        for duan in seg:
            for i in range(32):
                bbit=(duan>>i)%2
                if bbit==1:
                    mark_num+=1
    print '纪录了%d个记录'%mark_num
check_url_mark_num()
cur.close()

cxn.close()

print 'END'





